package com.hyt.it.ogt.kq.service.gov.crawer;

import com.hyt.it.ogt.kq.common.gov.constant.CodeEnum;
import com.hyt.it.ogt.kq.service.gov.crawer.entity.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;

import java.io.IOException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;

/**
 * 爬取国家统计局行政区划数据
 * （无需更新区域信息的情况下请勿执行此爬虫，需要更新区域信息时请先确认原先的区域数据已经清空 且国家统计局url为最新的url 之后才能执行此爬虫）
 */
@Slf4j
@Component
public class AreaCrawer {

    /**
     * 出错休息时间
     */
    private final int T2 = 1000;

    /**
     * 爬取国家统计局行政区划数据
     * @throws IOException IO异常
     * @throws InterruptedException InterruptedException
     */
    public List<RegionTable> AreaCrawling() throws IOException, InterruptedException {
        String listUrl = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/index.html";
        List<Province> province = getProvince(listUrl);
        List<RegionTable> nations = new ArrayList<>();
        RegionTable nation;
        for (Province value : province) {
            nation = new RegionTable();
            nation.setCode(value.getCode());
            nation.setName(value.getName());
            nation.setType(String.valueOf(CodeEnum.AREA_TYPE_PROVINCE.getCode()));
            nation.setParentCode("0");
            nations.add(nation);
            List<City> cities = value.getCities();
            if (cities != null) {
                for (City city : cities) {
                    nation = new RegionTable();
                    nation.setCode(city.getCode());
                    nation.setName(city.getName());
                    nation.setType(String.valueOf(CodeEnum.AREA_TYPE_CITY.getCode()));
                    nation.setParentCode(value.getCode());
                    nations.add(nation);
                    List<District> districts = city.getDistricts();
                    if (districts != null) {
                        for (District district : districts) {
                            nation = new RegionTable();
                            nation.setCode(district.getCode());
                            nation.setName(district.getName());
                            nation.setType(String.valueOf(CodeEnum.AREA_TYPE_DISTRICT.getCode()));
                            nation.setParentCode(city.getCode());
                            nations.add(nation);
                        }
                    }
                }
            }
        }

        //组装parentCodes数据
        for (RegionTable regionTable : nations) {
            List<String> parentCodes = new ArrayList<>();
            this.processParentCodes(regionTable, nations, parentCodes);
            if (!CollectionUtils.isEmpty(parentCodes)) {
                Collections.reverse(parentCodes);
                String parentCodesStr = String.join(",", parentCodes);
                regionTable.setParentCodes(parentCodesStr);
            }
        }
        //数据可以插入数据库或直接写入excel文件中，按实际情况处理
        return nations;
    }

    /**
     * 递归获取所有父级区域编号
     * @param regionTable 区域信息
     * @param nations 所有区域信息
     * @param parentCodes 所有父级区域编号
     */
    public void processParentCodes(RegionTable regionTable, List<RegionTable> nations, List<String> parentCodes) {
        String parentCode = regionTable.getParentCode();
        if (StringUtils.isNotEmpty(parentCode)) {
            parentCodes.add(parentCode);
            nations.forEach(n -> {
                if (parentCode.equals(n.getCode())) {
                    processParentCodes(n, nations, parentCodes);
                }
            });
        }
    }

    /**
     * 获取省级数据
     * @param listUrl
     * @return
     * @throws InterruptedException
     * @throws IOException
     */
    private List<Province> getProvince(final String listUrl) throws InterruptedException, IOException {
        Document doc;
        try {
            doc = Jsoup.parse(new URL(listUrl).openStream(), "GBK", listUrl);
        } catch (IOException e) {
            Thread.sleep(T2);
            //调用重新载入方法
            doc = heavyLoadIn(listUrl);
        }
        List<Province> provinces = new ArrayList<>();
        if(null != doc) {
        	//省级
        	Elements province = doc.select(".provincetr");
        	//查询市
            Province prov;
            for (Element item : province) {
                Elements a = item.getElementsByTag("a");
                for (Element value : a) {
                    prov = new Province();
                    //得到所有a标签
                    //a标签获取code
                    String href = value.attr("href");
                    //调用转换方法
                    href = getUTF8BytesFromGBKString(href);
                    String code = href.substring(0, href.lastIndexOf('.'));
                    prov.setCode(code);
                    //a标签获取name
                    String name = value.text();
                    //调用转换方法
                    name = getUTF8BytesFromGBKString(name);
                    prov.setName(name);
                    System.out.println(value.text());
                    //a标签获取地址获取市
                    List<City> cities = indexCity(value.absUrl("href"));
                    prov.setCities(cities);
                    provinces.add(prov);
                }
            }
        }
        return provinces;
    }

    /**
     * 获取市级数据
     * @param listUrl
     * @return
     * @throws InterruptedException
     * @throws IOException
     */
    private List<City> indexCity(String listUrl) throws InterruptedException, IOException {
        Document doc;
        try {
            //Thread.sleep(T);
            doc = Jsoup.parse(new URL(listUrl).openStream(), "GBK", listUrl);
//            doc = Jsoup.connect(listUrl).userAgent("Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36").timeout(T1).get();
        } catch (IOException e) {
            System.err.println("indexCity，重新载入");
            Thread.sleep(T2);
            //调用重新载入方法
            doc = heavyLoadIn(listUrl);
        }
        Elements city = Optional.ofNullable(doc).map(document -> document.select(".citytr")).orElseGet(Elements::new);

        //查询县
        List<City> cities = new ArrayList<>();
        City city1;
        for (Element value : city) {
            Elements a = value.getElementsByTag("a");
            for (int i1 = 0; i1 < a.size(); i1++) {
                city1 = new City();
                //得到所有a标签
                Element element = a.get(i1);
                //第一个标签为code第二个为名称
                //a标签获取code
                String code = element.text();
                //调用转换方法
                code = getUTF8BytesFromGBKString(code);
                city1.setCode(code);
                //a标签获取name
                String name = a.get(++i1).text();
                //调用转换方法
                name = getUTF8BytesFromGBKString(name);
                city1.setName(name);
                System.out.println(name);
                //a标签获取地址获取县
                List<District> counties = indexDistrict(element.absUrl("href"));
                city1.setDistricts(counties);
                cities.add(city1);
            }
        }
        return cities;
    }

    /**
     * 获取区县级数据
     * @param listUrl
     * @return
     * @throws InterruptedException
     * @throws IOException
     */
    private List<District> indexDistrict(String listUrl) throws InterruptedException, IOException {
        Document doc;
        try {
            doc = Jsoup.parse(new URL(listUrl).openStream(), "GBK", listUrl);
        } catch (IOException e) {
            Thread.sleep(T2);
            //调用重新载入方法
            doc = heavyLoadIn(listUrl);
        }
        List<District> counties = new ArrayList<>();
        if(null != doc) {
        	Elements county = doc.select(".countytr");//县
            //查询镇
            District district1;
            for (Element value : county) {
                Elements a = value.getElementsByTag("a");
                for (int i1 = 0; i1 < a.size(); i1++) {
                    district1 = new District();
                    //得到所有a标签
                    Element element = a.get(i1);
                    //第一个标签为code第二个为名称
                    //a标签获取code
                    String code = element.text();
                    //调用转换方法
                    code = getUTF8BytesFromGBKString(code);
                    district1.setCode(code);
                    //a标签获取name
                    String name = a.get(++i1).text();
                    //调用转换方法
                    name = getUTF8BytesFromGBKString(name);
                    district1.setName(name);
                    System.out.println(name);
                    //a标签获取地址获取镇
                    counties.add(district1);
                }
            }
        }
        return counties;
    }

    /**
     * 获取镇级数据
     * @param listUrl
     * @return
     * @throws InterruptedException
     * @throws IOException
     */
    private List<Country> indexCountry(String listUrl) throws InterruptedException, IOException {
        Document doc;
        try {
            doc = Jsoup.parse(new URL(listUrl).openStream(), "GBK", listUrl);
        } catch (IOException e) {
            Thread.sleep(T2);
            //调用重新载入方法
            doc = heavyLoadIn(listUrl);
        }
        List<Country> countries = new ArrayList<>();
        if(null != doc) {
        	Elements country = null;//街道
            try {
                country = Optional.ofNullable(doc).map(document -> document.select(".towntr")).orElseGet(Elements::new);
            } catch (Exception e) {
                //调用重新载入方法
                doc = heavyLoadIn(listUrl);
                country = Optional.ofNullable(doc).map(document -> document.select(".towntr")).orElseGet(Elements::new);
            }
            //查询村
            Country country1;
            for (Element value : country) {
                Elements a = value.getElementsByTag("a");
                for (int i1 = 0; i1 < a.size(); i1++) {
                    country1 = new Country();
                    //得到所有a标签
                    Element element = a.get(i1);
                    //第一个标签为code第二个为名称
                    //a标签获取code
                    String code = element.text();
                    //调用转换方法
                    code = getUTF8BytesFromGBKString(code);
                    country1.setCode(code);
                    //a标签获取name
                    Element elementName = a.get(++i1);
                    String name = elementName.text();
                    //调用转换方法
                    name = getUTF8BytesFromGBKString(name);
                    country1.setName(name);
                    System.out.println(name);
                    //a标签获取地址获取村
                    List<Town> towns = indexTown(element.absUrl("href"));
                    country1.setTowns(towns);
                    countries.add(country1);
                }
            }
        }
        return countries;
    }

    /**
     * 获取村级数据
     * @param listUrl
     * @return
     * @throws InterruptedException
     * @throws IOException
     */
    private List<Town> indexTown(String listUrl) throws InterruptedException, IOException {
        Document doc;
        try {
            doc = Jsoup.parse(new URL(listUrl).openStream(), "GBK", listUrl);
        } catch (IOException e) {
            //如果报错休息一段时间继续
            Thread.sleep(T2);
            //调用重新载入方法
            doc = heavyLoadIn(listUrl);
        }
        Elements town;
        try {
            town = Optional.ofNullable(doc).map(document -> document.select(".villagetr")).orElseGet(Elements::new);
        } catch (Exception e) {
            //调用重新载入方法
            doc = heavyLoadIn(listUrl);
            town = Optional.ofNullable(doc).map(document -> document.select(".villagetr")).orElseGet(Elements::new);
        }
        //查询村
        List<Town> towns = new ArrayList<>();
        Town town1;
        for (Element element : town) {
            town1 = new Town();
            Elements td = element.getElementsByTag("td");
            String code = td.get(0).text();
            code = getUTF8BytesFromGBKString(code);
            town1.setCode(code);
            String classification = td.get(1).text();
            classification = getUTF8BytesFromGBKString(classification);
            town1.setClassification(classification);
            String name = td.get(2).text();
            name = getUTF8BytesFromGBKString(name);
            town1.setName(name);
            towns.add(town1);
        }
        return towns;
    }

    private Document heavyLoadIn(String listUrl) {
        try {
            return Jsoup.parse(new URL(listUrl).openStream(), "GBK", listUrl);
        } catch (IOException e) {
            heavyLoadIn(listUrl);
        }
        return null;
    }

    /**
     * 有损转换
     * @param gbkStr
     * @return
     */
    public String getUTF8BytesFromGBKString(String gbkStr) {
        int n = gbkStr.length();
        byte[] utfBytes = new byte[3 * n];
        int k = 0;
        for (int i = 0; i < n; i++) {
            int m = gbkStr.charAt(i);
            if (m < 128 && m >= 0) {
                utfBytes[k++] = (byte) m;
                continue;
            }
            utfBytes[k++] = (byte) (0xe0 | (m >> 12));
            utfBytes[k++] = (byte) (0x80 | ((m >> 6) & 0x3f));
            utfBytes[k++] = (byte) (0x80 | (m & 0x3f));
        }
        if (k < utfBytes.length) {
            byte[] tmp = new byte[k];
            System.arraycopy(utfBytes, 0, tmp, 0, k);
            utfBytes = tmp;

        }
        return new String(utfBytes, StandardCharsets.UTF_8);
    }

}
